{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 模型"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 回归"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 线性回归调用方法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import mean_squared_error  #评价指标\n",
    "\n",
    "#从sklearn算法库中导入线性回归模型算法\n",
    "from sklearn.linear_model import LinearRegression\n",
    "clf = LinearRegression()\n",
    "clf.fit(train_data, train_target)\n",
    "test_pred = clf.predict(test_data)\n",
    "score = mean_squared_error(test_target, clf.predict(test_data))\n",
    "print(\"LinearRegression:   \", score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### K近邻回归调用方法"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$设有2个点P和Q,其中P={p_1,p_2,p_3,...p_n},Q={q_1,q_2,q_3,...q_n},n=1,2,3...\\\\\n",
    "那么P与Q之间的距离表示为d，则\\\\\n",
    "d=\\sqrt{{(p_1-q_1)^2+(p_2-q_2)^2+...+(P_n-q_n)^2}}$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 决策树回归调用方法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#从sklearn算法库中导入决策回归树模型算法\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "\n",
    "clf = DecisionTreeRegressor()\n",
    "clf.fit(train_data, train_target)\n",
    "test_pred = clf.predict(test_data)\n",
    "score = mean_squared_error(test_target, clf.predict(test_data))\n",
    "print(\"DecisionTreeRegressor:   \", score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 随机森林回归"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#从sklearn算法库中导入随机森林回归树模型算法\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "clf = RandomForestRegressor(n_estimators=200) # 200棵树模型\n",
    "clf.fit(train_data, train_target)\n",
    "test_pred = clf.predict(test_data)\n",
    "score = mean_squared_error(test_target, clf.predict(test_data))\n",
    "print(\"RandomForestRegressor:   \", score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 封装 Pipeline 管道Degree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "\n",
    "def PolynomialRegression(degree):\n",
    "    return Pipeline([('poly', PolynomialFeatures(degree=degree)),\n",
    "                     ('std_scaler', StandardScaler()),\n",
    "                     ('lin_reg', LinearRegression())])\n",
    "\n",
    "poly2_reg = PolynomialRegression(degree=2)\n",
    "poly2_reg.fit(X, y)\n",
    "y2_predict = poly2_reg.predict(X)\n",
    "# 比较真值和预测值的均方误差\n",
    "mean_squared_error(y, y2_predict)\n",
    "# 输出：1.0987392142417856"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LGB模型回归"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lgb回归模型\n",
    "clf = lgb.LGBMRegressor(\n",
    "    learning_rate=0.01,\n",
    "    max_depth=-1,\n",
    "    n_estimators=5000,\n",
    "    boosting_type='gbdt',\n",
    "    random_state=2019,\n",
    "    objective='regression',\n",
    ")\n",
    "\n",
    "# 训练模型\n",
    "clf.fit(X=train_data, y=train_target, eval_metric='MSE', verbose=50)\n",
    "\n",
    "score = mean_squared_error(test_target, clf.predict(test_data))\n",
    "print(\"lightGbm:   \", score)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 软投票简单示例"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-04T08:35:34.982633Z",
     "start_time": "2020-11-04T08:35:31.534177Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.gridspec as gridspec\n",
    "import itertools\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "##主要使用pip install mlxtend安装mlxtend\n",
    "from mlxtend.classifier import EnsembleVoteClassifier\n",
    "from mlxtend.data import iris_data\n",
    "from mlxtend.plotting import plot_decision_regions\n",
    "%matplotlib inline\n",
    "\n",
    "# Initializing Classifiers\n",
    "clf1 = LogisticRegression(random_state=0, \n",
    "                          solver='lbfgs', multi_class='auto')\n",
    "clf2 = RandomForestClassifier(random_state=0, n_estimators=100)\n",
    "clf3 = SVC(random_state=0, probability=True, gamma='auto')\n",
    "eclf = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3],\n",
    "                              weights=[2, 1, 1],\n",
    "                              voting='soft')\n",
    "\n",
    "# Loading some example data\n",
    "X, y = iris_data()\n",
    "X = X[:, [0, 2]]\n",
    "\n",
    "# Plotting Decision Regions\n",
    "gs = gridspec.GridSpec(1, 4)\n",
    "fig = plt.figure(figsize=(16, 4))\n",
    "\n",
    "for clf, lab, grd in zip(\n",
    "    [clf1, clf2, clf3, eclf],\n",
    "    ['Logistic Regression', 'Random Forest', \n",
    "     'RBF kernel SVM', 'Ensemble'],\n",
    "        itertools.product([0, 1], repeat=2)):\n",
    "    clf.fit(X, y)\n",
    "    ax = plt.subplot(gs[0, grd[0] * 2 + grd[1]])\n",
    "    fig = plot_decision_regions(X=X, y=y, clf=clf, legend=2)\n",
    "    plt.title(lab)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Stacking 回归\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from scipy import sparse\n",
    "import xgboost\n",
    "import lightgbm\n",
    "\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.ensemble import AdaBoostRegressor\n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "from sklearn.ensemble import ExtraTreesRegressor\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.metrics import mean_squared_error\n",
    "\n",
    "\n",
    "def stacking_reg(clf, train_x, train_y, test_x, clf_name, kf,\n",
    "                 label_split=None):\n",
    "    train = np.zeros((train_x.shape[0], 1))\n",
    "    test = np.zeros((test_x.shape[0], 1))\n",
    "    test_pre = np.empty((folds, test_x.shape[0], 1))\n",
    "    cv_scores = []\n",
    "    for i, (train_index,\n",
    "            test_index) in enumerate(kf.split(train_x, label_split)):\n",
    "        tr_x = train_x[train_index]\n",
    "        tr_y = train_y[train_index]\n",
    "        te_x = train_x[test_index]\n",
    "        te_y = train_y[test_index]\n",
    "        if clf_name in [\"rf\", \"ada\", \"gb\", \"et\", \"lr\", \"lsvc\", \"knn\"]:\n",
    "            clf.fit(tr_x, tr_y)\n",
    "            pre = clf.predict(te_x).reshape(-1, 1)\n",
    "            train[test_index] = pre\n",
    "            test_pre[i, :] = clf.predict(test_x).reshape(-1, 1)\n",
    "            cv_scores.append(mean_squared_error(te_y, pre))\n",
    "        elif clf_name in [\"xgb\"]:\n",
    "            train_matrix = clf.DMatrix(tr_x, label=tr_y, missing=-1)\n",
    "            test_matrix = clf.DMatrix(te_x, label=te_y, missing=-1)\n",
    "            z = clf.DMatrix(test_x, label=te_y, missing=-1)\n",
    "            params = {\n",
    "                'booster': 'gbtree',\n",
    "                'eval_metric': 'rmse',\n",
    "                'gamma': 1,\n",
    "                'min_child_weight': 1.5,\n",
    "                'max_depth': 5,\n",
    "                'lambda': 10,\n",
    "                'subsample': 0.7,\n",
    "                'colsample_bytree': 0.7,\n",
    "                'colsample_bylevel': 0.7,\n",
    "                'eta': 0.03,\n",
    "                'tree_method': 'exact',\n",
    "                'seed': 2017,\n",
    "                'nthread': 12\n",
    "            }\n",
    "            num_round = 10000\n",
    "            early_stopping_rounds = 100\n",
    "            watchlist = [(train_matrix, 'train'), (test_matrix, 'eval')]\n",
    "            if test_matrix:\n",
    "                model = clf.train(\n",
    "                    params,\n",
    "                    train_matrix,\n",
    "                    num_boost_round=num_round,\n",
    "                    evals=watchlist,\n",
    "                    early_stopping_rounds=early_stopping_rounds)\n",
    "                pre = model.predict(\n",
    "                    test_matrix,\n",
    "                    ntree_limit=model.best_ntree_limit).reshape(-1, 1)\n",
    "                train[test_index] = pre\n",
    "                test_pre[i, :] = model.predict(\n",
    "                    z, ntree_limit=model.best_ntree_limit).reshape(-1, 1)\n",
    "                cv_scores.append(mean_squared_error(te_y, pre))\n",
    "\n",
    "        elif clf_name in [\"lgb\"]:\n",
    "            train_matrix = clf.Dataset(tr_x, label=tr_y)\n",
    "            test_matrix = clf.Dataset(te_x, label=te_y)\n",
    "            params = {\n",
    "                'boosting_type': 'gbdt',\n",
    "                'objective': 'regression_l2',\n",
    "                'metric': 'mse',\n",
    "                'min_child_weight': 1.5,\n",
    "                'num_leaves': 2**5,\n",
    "                'lambda_l2': 10,\n",
    "                'subsample': 0.7,\n",
    "                'colsample_bytree': 0.7,\n",
    "                'colsample_bylevel': 0.7,\n",
    "                'learning_rate': 0.03,\n",
    "                'tree_method': 'exact',\n",
    "                'seed': 2017,\n",
    "                'nthread': 12,\n",
    "                'silent': True,\n",
    "            }\n",
    "            num_round = 10000\n",
    "            early_stopping_rounds = 100\n",
    "            if test_matrix:\n",
    "                model = clf.train(\n",
    "                    params,\n",
    "                    train_matrix,\n",
    "                    num_round,\n",
    "                    valid_sets=test_matrix,\n",
    "                    early_stopping_rounds=early_stopping_rounds)\n",
    "                pre = model.predict(\n",
    "                    te_x, \n",
    "                    num_iteration=model.best_iteration).reshape(-1, 1)\n",
    "                train[test_index] = pre\n",
    "                test_pre[i, :] = model.predict(\n",
    "                    test_x, \n",
    "                    num_iteration=model.best_iteration).reshape(-1, 1)\n",
    "                cv_scores.append(mean_squared_error(te_y, pre))\n",
    "        else:\n",
    "            raise IOError(\"Please add new clf.\")\n",
    "        print(\"%s now score is:\" % clf_name, cv_scores)\n",
    "    test[:] = test_pre.mean(axis=0)\n",
    "    print(\"%s_score_list:\" % clf_name, cv_scores)\n",
    "    print(\"%s_score_mean:\" % clf_name, np.mean(cv_scores))\n",
    "    return train.reshape(-1, 1), test.reshape(-1, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-11-04T10:28:54.923803Z",
     "start_time": "2020-11-04T10:28:54.912833Z"
    }
   },
   "outputs": [],
   "source": [
    "def rf_reg(x_train, y_train, x_valid, kf, label_split=None):\n",
    "    randomforest = RandomForestRegressor(n_estimators=600,\n",
    "                                         max_depth=20,\n",
    "                                         n_jobs=-1,\n",
    "                                         random_state=2017,\n",
    "                                         max_features=\"auto\",\n",
    "                                         verbose=1)\n",
    "    rf_train, rf_test = stacking_reg(randomforest,\n",
    "                                     x_train,\n",
    "                                     y_train,\n",
    "                                     x_valid,\n",
    "                                     \"rf\",\n",
    "                                     kf,\n",
    "                                     label_split=label_split)\n",
    "    return rf_train, rf_test, \"rf_reg\"\n",
    "\n",
    "\n",
    "def ada_reg(x_train, y_train, x_valid, kf, label_split=None):\n",
    "    adaboost = AdaBoostRegressor(n_estimators=30,\n",
    "                                 random_state=2017,\n",
    "                                 learning_rate=0.01)\n",
    "    ada_train, ada_test = stacking_reg(adaboost,\n",
    "                                       x_train,\n",
    "                                       y_train,\n",
    "                                       x_valid,\n",
    "                                       \"ada\",\n",
    "                                       kf,\n",
    "                                       label_split=label_split)\n",
    "    return ada_train, ada_test, \"ada_reg\"\n",
    "\n",
    "\n",
    "def gb_reg(x_train, y_train, x_valid, kf, label_split=None):\n",
    "    gbdt = GradientBoostingRegressor(learning_rate=0.04,\n",
    "                                     n_estimators=100,\n",
    "                                     subsample=0.8,\n",
    "                                     random_state=2017,\n",
    "                                     max_depth=5,\n",
    "                                     verbose=1)\n",
    "    gbdt_train, gbdt_test = stacking_reg(gbdt,\n",
    "                                         x_train,\n",
    "                                         y_train,\n",
    "                                         x_valid,\n",
    "                                         \"gb\",\n",
    "                                         kf,\n",
    "                                         label_split=label_split)\n",
    "    return gbdt_train, gbdt_test, \"gb_reg\"\n",
    "\n",
    "\n",
    "def et_reg(x_train, y_train, x_valid, kf, label_split=None):\n",
    "    extratree = ExtraTreesRegressor(n_estimators=600,\n",
    "                                    max_depth=35,\n",
    "                                    max_features=\"auto\",\n",
    "                                    n_jobs=-1,\n",
    "                                    random_state=2017,\n",
    "                                    verbose=1)\n",
    "    et_train, et_test = stacking_reg(extratree,\n",
    "                                     x_train,\n",
    "                                     y_train,\n",
    "                                     x_valid,\n",
    "                                     \"et\",\n",
    "                                     kf,\n",
    "                                     label_split=label_split)\n",
    "    return et_train, et_test, \"et_reg\"\n",
    "\n",
    "\n",
    "def lr_reg(x_train, y_train, x_valid, kf, label_split=None):\n",
    "    lr_reg = LinearRegression(n_jobs=-1)\n",
    "    lr_train, lr_test = stacking_reg(lr_reg,\n",
    "                                     x_train,\n",
    "                                     y_train,\n",
    "                                     x_valid,\n",
    "                                     \"lr\",\n",
    "                                     kf,\n",
    "                                     label_split=label_split)\n",
    "    return lr_train, lr_test, \"lr_reg\"\n",
    "\n",
    "\n",
    "def xgb_reg(x_train, y_train, x_valid, kf, label_split=None):\n",
    "    xgb_train, xgb_test = stacking_reg(xgboost,\n",
    "                                       x_train,\n",
    "                                       y_train,\n",
    "                                       x_valid,\n",
    "                                       \"xgb\",\n",
    "                                       kf,\n",
    "                                       label_split=label_split)\n",
    "    return xgb_train, xgb_test, \"xgb_reg\"\n",
    "\n",
    "\n",
    "def lgb_reg(x_train, y_train, x_valid, kf, label_split=None):\n",
    "    lgb_train, lgb_test = stacking_reg(lightgbm,\n",
    "                                       x_train,\n",
    "                                       y_train,\n",
    "                                       x_valid,\n",
    "                                       \"lgb\",\n",
    "                                       kf,\n",
    "                                       label_split=label_split)\n",
    "    return lgb_train, lgb_test, \"lgb_reg\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def stacking_pred(\n",
    "    x_train,y_train,\n",
    "    x_valid,kf,\n",
    "    clf_list,label_split=None,\n",
    "    clf_fin=\"lgb\",\n",
    "    if_concat_origin=True):\n",
    "    for k, clf_list in enumerate(clf_list):\n",
    "        clf_list = [clf_list]\n",
    "        column_list = []\n",
    "        train_data_list = []\n",
    "        test_data_list = []\n",
    "        for clf in clf_list:\n",
    "            train_data, test_data, clf_name = clf(\n",
    "                x_train,\n",
    "                y_train,\n",
    "                x_valid,\n",
    "                kf,\n",
    "                label_split=label_split)\n",
    "            train_data_list.append(train_data)\n",
    "            test_data_list.append(test_data)\n",
    "            column_list.append(\"clf_%s\" % (clf_name))\n",
    "    train = np.concatenate(train_data_list, axis=1)\n",
    "    test = np.concatenate(test_data_list, axis=1)\n",
    "\n",
    "    if if_concat_origin:\n",
    "        train = np.concatenate([x_train, train], axis=1)\n",
    "        test = np.concatenate([x_valid, test], axis=1)\n",
    "    print(x_train.shape)\n",
    "    print(train.shape)\n",
    "    print(clf_name)\n",
    "    print(clf_name in [\"lgb\"])\n",
    "    if clf_fin in [\"rf\", \"ada\", \"gb\", \"et\", \"lr\", \"lsvc\", \"knn\"]:\n",
    "        if clf_fin in [\"rf\"]:\n",
    "            clf = RandomForestRegressor(\n",
    "                n_estimators=600,\n",
    "                max_depth=20,\n",
    "                n_jobs=-1,\n",
    "                random_state=2017,\n",
    "                max_features=\"auto\",\n",
    "                verbose=1)\n",
    "        elif clf_fin in [\"ada\"]:\n",
    "            clf = AdaBoostRegressor(\n",
    "                n_estimators=30,\n",
    "                random_state=2017,\n",
    "                learning_rate=0.01)\n",
    "        elif clf_fin in [\"gb\"]:\n",
    "            clf = GradientBoostingRegressor(\n",
    "                learning_rate=0.04,\n",
    "                n_estimators=100,\n",
    "                subsample=0.8,\n",
    "                random_state=2017,\n",
    "                max_depth=5,\n",
    "                verbose=1)\n",
    "        elif clf_fin in [\"et\"]:\n",
    "            clf = ExtraTreesRegressor(\n",
    "                n_estimators=600,\n",
    "                max_depth=35,\n",
    "                max_features=\"auto\",\n",
    "                n_jobs=-1,\n",
    "                random_state=2017,\n",
    "                verbose=1)\n",
    "        elif clf_fin in [\"lr\"]:\n",
    "            clf = LinearRegression(n_jobs=-1)\n",
    "        clf.fit(train, y_train)\n",
    "        pre = clf.predict(test).reshape(-1, 1)\n",
    "        return pred\n",
    "    elif clf_fin in [\"xgb\"]:\n",
    "        clf = xgboost\n",
    "        train_matrix = clf.DMatrix(train, \n",
    "                                   label=y_train, missing=-1)\n",
    "        test_matrix = clf.DMatrix(train, \n",
    "                                  label=y_train, missing=-1)\n",
    "        params = {\n",
    "            'booster': 'gbtree',\n",
    "            'eval_metric': 'rmse',\n",
    "            'gamma': 1,\n",
    "            'min_child_weight': 1.5,\n",
    "            'max_depth': 5,\n",
    "            'lambda': 10,\n",
    "            'subsample': 0.7,\n",
    "            'colsample_bytree': 0.7,\n",
    "            'colsample_bylevel': 0.7,\n",
    "            'eta': 0.03,\n",
    "            'tree_method': 'exact',\n",
    "            'seed': 2017,\n",
    "            'nthread': 12\n",
    "        }\n",
    "        num_round = 10000\n",
    "        early_stopping_rounds = 100\n",
    "        watchlist = [(train_matrix, 'train'), (test_matrix, 'eval')]\n",
    "        model = clf.train(\n",
    "            params,\n",
    "            train_matrix,\n",
    "            num_boost_round=num_round,\n",
    "            evals=watchlist,\n",
    "            early_stopping_rounds=early_stopping_rounds)\n",
    "        pre = model.predict(\n",
    "            test,\n",
    "            ntree_limit=model.best_ntree_limit).reshape(-1, 1)\n",
    "        return pre\n",
    "    elif clf_fin in [\"lgb\"]:\n",
    "        print(clf_name)\n",
    "        clf = lightgbm\n",
    "        train_matrix = clf.Dataset(train, label=y_train)\n",
    "        test_matrix = clf.Dataset(train, label=y_train)\n",
    "        params = {\n",
    "            'boosting_type': 'gbdt',\n",
    "            'objective': 'regression_l2',\n",
    "            'metric': 'mse',\n",
    "            'min_child_weight': 1.5,\n",
    "            'num_leaves': 2**5,\n",
    "            'lambda_l2': 10,\n",
    "            'subsample': 0.7,\n",
    "            'colsample_bytree': 0.7,\n",
    "            'colsample_bylevel': 0.7,\n",
    "            'learning_rate': 0.03,\n",
    "            'tree_method': 'exact',\n",
    "            'seed': 2017,\n",
    "            'nthread': 12,\n",
    "            'silent': True,\n",
    "        }\n",
    "        num_round = 10000\n",
    "        early_stopping_rounds = 100\n",
    "        model = clf.train(\n",
    "            params,\n",
    "            train_matrix,\n",
    "            num_round,\n",
    "            valid_sets=test_matrix,\n",
    "            early_stopping_rounds=early_stopping_rounds)\n",
    "        print('pred')\n",
    "        pre = model.predict(\n",
    "            test,\n",
    "            num_iteration=model.best_iteration).reshape(-1, 1)\n",
    "        print(pre)\n",
    "        return pre"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# K折交叉验证\n",
    "from sklearn.model_selection import StratifiedKFold, KFold\n",
    "folds = 5\n",
    "seed = 1\n",
    "kf = KFold(n_splits=5, shuffle=True, random_state=0)\n",
    "\n",
    "# 训练集和测试集数据\n",
    "x_train = data_train[data_test.columns].values\n",
    "x_valid = data_test[data_test.columns].values\n",
    "y_train = data_train['target'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 使用lr_reg和lgb_reg进行融合预测\n",
    "clf_list = [lr_reg, lgb_reg]\n",
    "\n",
    "##很容易过拟合\n",
    "pred = stacking_pred(x_train, y_train, x_valid, kf, clf_list, label_split=None, clf_fin=\"lgb\", if_concat_origin=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 分类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import metrics  \n",
    "import numpy as np  \n",
    "import pandas as pd\n",
    "import datetime\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "import lightgbm as lgb\n",
    "import xgboost as xgb\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.model_selection import learning_curve\n",
    "from sklearn.model_selection import ShuffleSplit\n",
    "#########SKLearn 集成的算法###############\n",
    "from sklearn.neighbors import KNeighborsClassifier  \n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn import tree  \n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.svm import SVC \n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from xgboost import XGBClassifier\n",
    "from lightgbm import LGBMClassifier\n",
    "from sklearn.naive_bayes import MultinomialNB \n",
    "#########SKLearn 集成的算法###############\n",
    "from sklearn.model_selection import train_test_split # 切分数据\n",
    "from sklearn.metrics import mean_squared_error #评价指标\n",
    "\n",
    "from sklearn.model_selection import learning_curve\n",
    "from sklearn.model_selection import ShuffleSplit\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    " \n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "\n",
    "####################部分整合在sklearn的分类算法###############\n",
    "def get_sklearn_model(model_name):\n",
    "    #朴素贝叶斯\n",
    "    if model_name == 'NB':\n",
    "        return MultinomialNB(alpha=0.01)\n",
    "    #逻辑回归\n",
    "    elif model_name == 'LR':\n",
    "        return LogisticRegression(penalty='l2')\n",
    "    # KNN\n",
    "    elif model_name == 'KNN':\n",
    "        return KNeighborsClassifier()\n",
    "    #随机森林\n",
    "    elif model_name == 'RF':\n",
    "        return RandomForestClassifier()\n",
    "    #决策树\n",
    "    elif model_name == 'DT':\n",
    "        return tree.DecisionTreeClassifier()\n",
    "    #向量机\n",
    "    elif model_name == 'SVC':\n",
    "        return SVC(kernel='rbf')\n",
    "    #GBDT\n",
    "    elif model_name == 'GBDT':\n",
    "        return GradientBoostingClassifier()\n",
    "    #XGBoost\n",
    "    elif model_name == 'XGB':\n",
    "        return XGBClassifier()\n",
    "    #lightGBM\n",
    "    elif model_name == 'LGB':\n",
    "        return LGBMClassifier()\n",
    "    else:\n",
    "        print(\"wrong model name!\")\n",
    "\n",
    "\n",
    "#画学习曲线\n",
    "def plot_learning_curve(estimator,\n",
    "                        title,\n",
    "                        X,\n",
    "                        y,\n",
    "                        ylim=None,\n",
    "                        cv=None,\n",
    "                        n_jobs=1,\n",
    "                        train_sizes=[0.01, 0.02, 0.05, 0.1, 0.2, 0.3]):\n",
    "    plt.figure()\n",
    "    plt.title(title)\n",
    "    if ylim is not None:\n",
    "        plt.ylim(*ylim)\n",
    "    plt.xlabel(\"Training examples\")\n",
    "    plt.ylabel(\"Score\")\n",
    "    train_sizes, train_scores, test_scores = learning_curve(\n",
    "        estimator,\n",
    "        X,\n",
    "        y,\n",
    "        cv=cv,\n",
    "        scoring=myeval,\n",
    "        n_jobs=n_jobs,\n",
    "        train_sizes=train_sizes)\n",
    "    train_scores_mean = np.mean(train_scores, axis=1)\n",
    "    train_scores_std = np.std(train_scores, axis=1)\n",
    "    test_scores_mean = np.mean(test_scores, axis=1)\n",
    "    test_scores_std = np.std(test_scores, axis=1)\n",
    "    plt.grid()\n",
    "\n",
    "    plt.fill_between(train_sizes,\n",
    "                     train_scores_mean - train_scores_std,\n",
    "                     train_scores_mean + train_scores_std,\n",
    "                     alpha=0.1,\n",
    "                     color=\"r\")\n",
    "    plt.fill_between(train_sizes,\n",
    "                     test_scores_mean - test_scores_std,\n",
    "                     test_scores_mean + test_scores_std,\n",
    "                     alpha=0.1,\n",
    "                     color=\"g\")\n",
    "    plt.plot(train_sizes,\n",
    "             train_scores_mean,\n",
    "             'o-',\n",
    "             color=\"r\",\n",
    "             label=\"Training score\")\n",
    "    plt.plot(train_sizes,\n",
    "             test_scores_mean,\n",
    "             'o-',\n",
    "             color=\"g\",\n",
    "             label=\"Cross-validation score\")\n",
    "\n",
    "    plt.legend(loc=\"best\")\n",
    "    return plt\n",
    "\n",
    "\n",
    "#画算法的学习曲线,为加快画图速度，最多选20%数据\n",
    "def plot_curve_single(traindf,\n",
    "                      classifier,\n",
    "                      cvnum,\n",
    "                      train_sizes=[0.01, 0.02, 0.05, 0.1, 0.2, 0.3]):\n",
    "    X = get_predictors_df(traindf)\n",
    "    y = get_target_df(traindf)\n",
    "    estimator = get_sklearn_model(classifier)  #建模\n",
    "    title = \"learning curve of \" + classifier + \", cv:\" + str(cvnum)\n",
    "    plot_learning_curve(estimator,\n",
    "                        title,\n",
    "                        X,\n",
    "                        y,\n",
    "                        ylim=(0, 1.01),\n",
    "                        cv=cvnum,\n",
    "                        train_sizes=train_sizes)\n",
    "\n",
    "\n",
    "#性能评价函数\n",
    "#本赛题目标是预测投放的优惠券是否核销。\n",
    "#针对此任务及一些相关背景知识，使用优惠券核销预测的平均AUC（ROC曲线下面积）作为评价标准。\n",
    "#即对每个优惠券coupon_id单独计算核销预测的AUC值，再对所有优惠券的AUC值求平均作为最终的评价标准。\n",
    "# coupon平均auc计算\n",
    "def myauc(test):\n",
    "    testgroup = test.groupby(['coupon_id'])\n",
    "    aucs = []\n",
    "    for i in testgroup:\n",
    "        coupon_df = i[1]\n",
    "        #测算AUC必须大于1个类别\n",
    "        if len(coupon_df['label'].unique()) < 2:\n",
    "            continue\n",
    "        auc = metrics.roc_auc_score(coupon_df['label'], coupon_df['pred'])\n",
    "        aucs.append(auc)\n",
    "    return np.average(aucs)\n",
    "\n",
    "\n",
    "#按照日期分割\n",
    "def test_model(traindf, classifier):\n",
    "    train = traindf[traindf.date_received < 20160515].copy()\n",
    "    test = traindf[traindf.date_received >= 20160515].copy()\n",
    "\n",
    "    train_data = get_predictors_df(train).copy()\n",
    "    train_target = get_target_df(train).copy()\n",
    "    test_data = get_predictors_df(test).copy()\n",
    "    test_target = get_target_df(test).copy()\n",
    "\n",
    "    clf = get_sklearn_model(classifier)\n",
    "    clf.fit(train_data, train_target)\n",
    "    result = clf.predict_proba(test_data)[:, 1]\n",
    "    test['pred'] = result\n",
    "    score = metrics.roc_auc_score(test_target, result)\n",
    "    print(classifier + \" 总体 AUC:\", score)\n",
    "    score_coupon = myauc(test)\n",
    "    print(classifier + \" Coupon AUC:\", score_coupon)\n",
    "\n",
    "\n",
    "def test_model_split(traindf, classifier):\n",
    "\n",
    "    target = get_target_df(traindf).copy()\n",
    "\n",
    "    train_all, test_all, train_target, test_target = train_test_split(\n",
    "        traindf, target, test_size=0.2, random_state=0)\n",
    "\n",
    "    train_data = get_predictors_df(train_all).copy()\n",
    "    test_data = get_predictors_df(test_all).copy()\n",
    "\n",
    "    clf = get_sklearn_model(classifier)\n",
    "    clf.fit(train_data, train_target)\n",
    "    result = clf.predict_proba(test_data)[:, 1]\n",
    "\n",
    "    test = test_all.copy()\n",
    "    test['pred'] = result\n",
    "\n",
    "    score = metrics.roc_auc_score(test_target, result)\n",
    "    print(classifier + \"总体AUC:\", score)\n",
    "    score_coupon = myauc(test)\n",
    "    print(classifier + \" Coupon AUC:\", score_coupon)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_model(train_f1, 'LR')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('LR Coupon AUC')\n",
    "plot_curve_single(train_f1, 'LR', 5, [0.01, 0.02, 0.05, 0.1, 0.2, 0.3])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LR 模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "stdScaler = StandardScaler()\n",
    "X = stdScaler.fit_transform(train)\n",
    "\n",
    "# Split the data into a training set and a test set\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, target, random_state=0)\n",
    "\n",
    "clf = LogisticRegression(random_state=0, solver='lbfgs', multi_class='multinomial').fit(X_train, y_train)\n",
    "clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### KNN 模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "stdScaler = StandardScaler()\n",
    "X = stdScaler.fit_transform(train)\n",
    "\n",
    "# Split the data into a training set and a test set\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, target, random_state=0)\n",
    "\n",
    "clf = KNeighborsClassifier(n_neighbors=3).fit(X_train, y_train)\n",
    "clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GaussianNB 模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "stdScaler = StandardScaler()\n",
    "X = stdScaler.fit_transform(train)\n",
    "\n",
    "# Split the data into a training set and a test set\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, target, random_state=0)\n",
    "\n",
    "clf = GaussianNB().fit(X_train, y_train)\n",
    "clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### tree树模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import tree\n",
    "\n",
    "# Split the data into a training set and a test set\n",
    "X_train, X_test, y_train, y_test = train_test_split(train, target, random_state=0)\n",
    "\n",
    "clf = tree.DecisionTreeClassifier()\n",
    "clf = clf.fit(X_train, y_train)\n",
    "clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### bagging模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import BaggingClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "# Split the data into a training set and a test set\n",
    "X_train, X_test, y_train, y_test = train_test_split(train, target, random_state=0)\n",
    "clf = BaggingClassifier(KNeighborsClassifier(), max_samples=0.5, max_features=0.5)\n",
    "\n",
    "clf = clf.fit(X_train, y_train)\n",
    "clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 随机森林模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "# Split the data into a training set and a test set\n",
    "X_train, X_test, y_train, y_test = train_test_split(train, target, random_state=0)\n",
    "clf = clf = RandomForestClassifier(n_estimators=10, max_depth=None, min_samples_split=2, random_state=0)\n",
    "\n",
    "clf = clf.fit(X_train, y_train)\n",
    "clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ExTree模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "\n",
    "# Split the data into a training set and a test set\n",
    "X_train, X_test, y_train, y_test = train_test_split(train, target, random_state=0)\n",
    "clf = ExtraTreesClassifier(n_estimators=10, max_depth=None, min_samples_split=2, random_state=0)\n",
    "\n",
    "clf = clf.fit(X_train, y_train)\n",
    "clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### AdaBoost模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "\n",
    "# Split the data into a training set and a test set\n",
    "X_train, X_test, y_train, y_test = train_test_split(train, target, random_state=0)\n",
    "clf = AdaBoostClassifier(n_estimators=100)\n",
    "\n",
    "clf = clf.fit(X_train, y_train)\n",
    "clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GBDT模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "\n",
    "# Split the data into a training set and a test set\n",
    "X_train, X_test, y_train, y_test = train_test_split(train, target, random_state=0)\n",
    "clf = GradientBoostingClassifier(n_estimators=100, learning_rate=1.0, max_depth=1, random_state=0)\n",
    "\n",
    "clf = clf.fit(X_train, y_train)\n",
    "clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### VOTE模型投票"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import datasets\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import VotingClassifier\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "stdScaler = StandardScaler()\n",
    "X = stdScaler.fit_transform(train)\n",
    "y = target\n",
    "\n",
    "\n",
    "clf1 = LogisticRegression(solver='lbfgs', multi_class='multinomial', random_state=1)\n",
    "clf2 = RandomForestClassifier(n_estimators=50, random_state=1)\n",
    "clf3 = GaussianNB()\n",
    "\n",
    "eclf = VotingClassifier(estimators=[('lr', clf1), ('rf', clf2), ('gnb', clf3)], voting='hard')\n",
    "\n",
    "for clf, label in zip([clf1, clf2, clf3, eclf], ['Logistic Regression', 'Random Forest', 'naive Bayes', 'Ensemble']):\n",
    "    scores = cross_val_score(clf, X, y, cv=5, scoring='accuracy')\n",
    "    print(\"Accuracy: %0.2f (+/- %0.2f) [%s]\" % (scores.mean(), scores.std(), label))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### lgb 模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import lightgbm\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(train, target, test_size=0.4, random_state=0)\n",
    "X_test, X_valid, y_test, y_valid = train_test_split(X_test, y_test, test_size=0.5, random_state=0)\n",
    "\n",
    "clf = lightgbm\n",
    "\n",
    "train_matrix = clf.Dataset(X_train, label=y_train)\n",
    "test_matrix = clf.Dataset(X_test, label=y_test)\n",
    "params = {\n",
    "          'boosting_type': 'gbdt',\n",
    "          #'boosting_type': 'dart',\n",
    "          'objective': 'multiclass',\n",
    "          'metric': 'multi_logloss',\n",
    "          'min_child_weight': 1.5,\n",
    "          'num_leaves': 2**5,\n",
    "          'lambda_l2': 10,\n",
    "          'subsample': 0.7,\n",
    "          'colsample_bytree': 0.7,\n",
    "          'colsample_bylevel': 0.7,\n",
    "          'learning_rate': 0.03,\n",
    "          'tree_method': 'exact',\n",
    "          'seed': 2017,\n",
    "          \"num_class\": 2,\n",
    "          'silent': True,\n",
    "          }\n",
    "num_round = 10000\n",
    "early_stopping_rounds = 100\n",
    "model = clf.train(params, \n",
    "                  train_matrix,\n",
    "                  num_round,\n",
    "                  valid_sets=test_matrix,\n",
    "                  early_stopping_rounds=early_stopping_rounds)\n",
    "pre= model.predict(X_valid,num_iteration=model.best_iteration)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('score : ', np.mean((pre[:,1]>0.5)==y_valid))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### xgb 模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xgboost\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(train, target, test_size=0.4, random_state=0)\n",
    "X_test, X_valid, y_test, y_valid = train_test_split(X_test, y_test, test_size=0.5, random_state=0)\n",
    "\n",
    "clf = xgboost\n",
    "\n",
    "train_matrix = clf.DMatrix(X_train, label=y_train, missing=-1)\n",
    "test_matrix = clf.DMatrix(X_test, label=y_test, missing=-1)\n",
    "z = clf.DMatrix(X_valid, label=y_valid, missing=-1)\n",
    "params = {'booster': 'gbtree',\n",
    "          'objective': 'multi:softprob',\n",
    "          'eval_metric': 'mlogloss',\n",
    "          'gamma': 1,\n",
    "          'min_child_weight': 1.5,\n",
    "          'max_depth': 5,\n",
    "          'lambda': 10,\n",
    "          'subsample': 0.7,\n",
    "          'colsample_bytree': 0.7,\n",
    "          'colsample_bylevel': 0.7,\n",
    "          'eta': 0.03,\n",
    "          'tree_method': 'exact',\n",
    "          'seed': 2017,\n",
    "          \"num_class\": 2\n",
    "          }\n",
    "\n",
    "num_round = 10000\n",
    "early_stopping_rounds = 100\n",
    "watchlist = [(train_matrix, 'train'),\n",
    "             (test_matrix, 'eval')\n",
    "             ]\n",
    "\n",
    "model = clf.train(params,\n",
    "                  train_matrix,\n",
    "                  num_boost_round=num_round,\n",
    "                  evals=watchlist,\n",
    "                  early_stopping_rounds=early_stopping_rounds\n",
    "                  )\n",
    "pre = model.predict(z,ntree_limit=model.best_ntree_limit)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('score : ', np.mean((pre[:,1]>0.5)==y_valid))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Stacking,Bootstrap,Bagging技术实践"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "    导入相关包\n",
    "\"\"\"\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import lightgbm as lgb\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "class SBBTree():\n",
    "    \"\"\"\n",
    "        SBBTree\n",
    "        Stacking,Bootstap,Bagging\n",
    "    \"\"\"\n",
    "    def __init__(\n",
    "                    self, \n",
    "                    params,\n",
    "                    stacking_num,\n",
    "                    bagging_num,\n",
    "                    bagging_test_size,\n",
    "                    num_boost_round,\n",
    "                    early_stopping_rounds\n",
    "                ):\n",
    "        \"\"\"\n",
    "            Initializes the SBBTree.\n",
    "            Args:\n",
    "              params : lgb params.\n",
    "              stacking_num : k_flod stacking.\n",
    "              bagging_num : bootstrap num.\n",
    "              bagging_test_size : bootstrap sample rate.\n",
    "              num_boost_round : boost num.\n",
    "              early_stopping_rounds : early_stopping_rounds.\n",
    "        \"\"\"\n",
    "        self.params = params\n",
    "        self.stacking_num = stacking_num\n",
    "        self.bagging_num = bagging_num\n",
    "        self.bagging_test_size = bagging_test_size\n",
    "        self.num_boost_round = num_boost_round\n",
    "        self.early_stopping_rounds = early_stopping_rounds\n",
    "\n",
    "        self.model = lgb\n",
    "        self.stacking_model = []\n",
    "        self.bagging_model = []\n",
    "\n",
    "    def fit(self, X, y):\n",
    "        \"\"\" fit model. \"\"\"\n",
    "        if self.stacking_num > 1:\n",
    "            layer_train = np.zeros((X.shape[0], 2))\n",
    "            self.SK = StratifiedKFold(n_splits=self.stacking_num, shuffle=True, random_state=1)\n",
    "            for k,(train_index, test_index) in enumerate(self.SK.split(X, y)):\n",
    "                X_train = X[train_index]\n",
    "                y_train = y[train_index]\n",
    "                X_test = X[test_index]\n",
    "                y_test = y[test_index]\n",
    "\n",
    "                lgb_train = lgb.Dataset(X_train, y_train)\n",
    "                lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)\n",
    "\n",
    "                gbm = lgb.train(self.params,\n",
    "                            lgb_train,\n",
    "                            num_boost_round=self.num_boost_round,\n",
    "                            valid_sets=lgb_eval,\n",
    "                            early_stopping_rounds=self.early_stopping_rounds)\n",
    "\n",
    "                self.stacking_model.append(gbm)\n",
    "\n",
    "                pred_y = gbm.predict(X_test, num_iteration=gbm.best_iteration)\n",
    "                layer_train[test_index, 1] = pred_y\n",
    "\n",
    "            X = np.hstack((X, layer_train[:,1].reshape((-1,1)))) \n",
    "        else:\n",
    "            pass\n",
    "        for bn in range(self.bagging_num):\n",
    "            X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=self.bagging_test_size, random_state=bn)\n",
    "\n",
    "            lgb_train = lgb.Dataset(X_train, y_train)\n",
    "            lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)\n",
    "\n",
    "            gbm = lgb.train(self.params,\n",
    "                        lgb_train,\n",
    "                        num_boost_round=10000,\n",
    "                        valid_sets=lgb_eval,\n",
    "                        early_stopping_rounds=200)\n",
    "\n",
    "            self.bagging_model.append(gbm)\n",
    "\n",
    "    def predict(self, X_pred):\n",
    "        \"\"\" predict test data. \"\"\"\n",
    "        if self.stacking_num > 1:\n",
    "            test_pred = np.zeros((X_pred.shape[0], self.stacking_num))\n",
    "            for sn,gbm in enumerate(self.stacking_model):\n",
    "                pred = gbm.predict(X_pred, num_iteration=gbm.best_iteration)\n",
    "                test_pred[:, sn] = pred\n",
    "            X_pred = np.hstack((X_pred, test_pred.mean(axis=1).reshape((-1,1))))  \n",
    "        else:\n",
    "            pass \n",
    "        for bn,gbm in enumerate(self.bagging_model):\n",
    "            pred = gbm.predict(X_pred, num_iteration=gbm.best_iteration)\n",
    "            if bn == 0:\n",
    "                pred_out=pred\n",
    "            else:\n",
    "                pred_out+=pred\n",
    "        return pred_out/self.bagging_num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import lightgbm as lgb\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "train_data = pd.read_csv('train_all.csv',nrows=10000)\n",
    "test_data = pd.read_csv('test_all.csv',nrows=100)\n",
    "\n",
    "features_columns = [col for col in train_data.columns if col not in ['user_id','label']]\n",
    "train = train_data[features_columns].values\n",
    "test = test_data[features_columns].values\n",
    "target =train_data['label'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {\n",
    "        'task': 'train',\n",
    "        'boosting_type': 'gbdt',\n",
    "        'objective': 'binary',\n",
    "        'metric': 'auc',\n",
    "        'num_leaves': 9,\n",
    "        'learning_rate': 0.03,\n",
    "        'feature_fraction_seed': 2,\n",
    "        'feature_fraction': 0.9,\n",
    "        'bagging_fraction': 0.8,\n",
    "        'bagging_freq': 5,\n",
    "        'min_data': 20,\n",
    "        'min_hessian': 1,\n",
    "        'verbose': -1,\n",
    "        'silent': 0\n",
    "        }\n",
    "\n",
    "model = SBBTree(params=params,\n",
    "                stacking_num=5,\n",
    "                bagging_num=3,\n",
    "                bagging_test_size=0.33,\n",
    "                num_boost_round=10000,\n",
    "                early_stopping_rounds=200)\n",
    "\n",
    "model.fit(train, target)\n",
    "\n",
    "pred = model.predict(test)\n",
    "df_out = pd.DataFrame()\n",
    "df_out['user_id'] = test_data['user_id']\n",
    "df_out['predict_prob'] = pred\n",
    "df_out.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 调参"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "\n",
    "# Split the dataset in two equal parts\n",
    "X_train, X_test, y_train, y_test = train_test_split(train, target, test_size=0.5, random_state=0)\n",
    "\n",
    "# model \n",
    "clf = RandomForestClassifier(n_jobs=-1)\n",
    "\n",
    "# Set the parameters by cross-validation\n",
    "\n",
    "tuned_parameters = {\n",
    "                    'n_estimators': [50, 100, 200]\n",
    "#                     ,'criterion': ['gini', 'entropy']\n",
    "#                     ,'max_depth': [2, 5]\n",
    "#                     ,'max_features': ['log2', 'sqrt', 'int']\n",
    "#                     ,'bootstrap': [True, False]\n",
    "#                     ,'warm_start': [True, False]\n",
    "                    }\n",
    "\n",
    "scores = ['precision']\n",
    "\n",
    "for score in scores:\n",
    "    print(\"# Tuning hyper-parameters for %s\" % score)\n",
    "    print()\n",
    "\n",
    "    clf = GridSearchCV(clf, tuned_parameters, cv=5,\n",
    "                       scoring='%s_macro' % score)\n",
    "    clf.fit(X_train, y_train)\n",
    "\n",
    "    print(\"Best parameters set found on development set:\")\n",
    "    print()\n",
    "    print(clf.best_params_)\n",
    "    print()\n",
    "    print(\"Grid scores on development set:\")\n",
    "    print()\n",
    "    means = clf.cv_results_['mean_test_score']\n",
    "    stds = clf.cv_results_['std_test_score']\n",
    "    for mean, std, params in zip(means, stds, clf.cv_results_['params']):\n",
    "        print(\"%0.3f (+/-%0.03f) for %r\"\n",
    "              % (mean, std * 2, params))\n",
    "    print()\n",
    "\n",
    "    print(\"Detailed classification report:\")\n",
    "    print()\n",
    "    print(\"The model is trained on the full development set.\")\n",
    "    print(\"The scores are computed on the full evaluation set.\")\n",
    "    print()\n",
    "    y_true, y_pred = y_test, clf.predict(X_test)\n",
    "    print(classification_report(y_true, y_pred))\n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 模型交叉验证"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "8分类问题"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 训练集&测试集构建\n",
    "train_data = train_label.merge(simple_train_fea1, on ='file_id', how='left')\n",
    "train_data = train_data.merge(simple_train_fea2, on ='file_id', how='left')\n",
    "train_data = train_data.merge(simple_train_fea3, on ='file_id', how='left')\n",
    "train_data = train_data.merge(simple_train_fea4, on ='file_id', how='left')\n",
    "\n",
    "test_submit = test_submit.merge(simple_test_fea1, on ='file_id', how='left')\n",
    "test_submit = test_submit.merge(simple_test_fea2, on ='file_id', how='left')\n",
    "test_submit = test_submit.merge(simple_test_fea3, on ='file_id', how='left')\n",
    "test_submit = test_submit.merge(simple_test_fea4, on ='file_id', how='left')\n",
    "\n",
    "train_features = [col for col in train_data.columns if col not in ['label','file_id']]\n",
    "train_label    = 'label'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 评估指标构建\n",
    "def lgb_logloss(preds,data):\n",
    "    labels_ = data.get_label()             \n",
    "    classes_ = np.unique(labels_) \n",
    "    preds_prob = []\n",
    "    for i in range(len(classes_)):\n",
    "        preds_prob.append(preds[i*len(labels_):(i+1) * len(labels_)] )\n",
    "        \n",
    "    preds_prob_ = np.vstack(preds_prob) \n",
    "    \n",
    "    loss = []\n",
    "    for i in range(preds_prob_.shape[1]):     \n",
    "        sum_ = 0\n",
    "        for j in range(preds_prob_.shape[0]): \n",
    "            pred = preds_prob_[j,i]           \n",
    "            if  j == labels_[i]:\n",
    "                sum_ += np.log(pred)\n",
    "            else:\n",
    "                sum_ += np.log(1 - pred)\n",
    "        loss.append(sum_)       \n",
    "    return 'loss is: ',-1 * (np.sum(loss) / preds_prob_.shape[1]),False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "from sklearn.model_selection import StratifiedKFold,KFold\n",
    "params = {\n",
    "        'task':'train', \n",
    "        'num_leaves': 255,\n",
    "        'objective': 'multiclass',\n",
    "        'num_class': 8,\n",
    "        'min_data_in_leaf': 50,\n",
    "        'learning_rate': 0.05,\n",
    "        'feature_fraction': 0.85,\n",
    "        'bagging_fraction': 0.85,\n",
    "        'bagging_freq': 5, \n",
    "        'max_bin':128,\n",
    "        'random_state':100\n",
    "    }   \n",
    "\n",
    "folds = KFold(n_splits=5, shuffle=True, random_state=15)\n",
    "oof = np.zeros(len(train))\n",
    "\n",
    "predict_res = 0\n",
    "models = []\n",
    "for fold_, (trn_idx, val_idx) in enumerate(folds.split(train_data)):\n",
    "    print(\"fold n°{}\".format(fold_))\n",
    "    trn_data = lgb.Dataset(train_data.iloc[trn_idx][train_features], label=train_data.iloc[trn_idx][train_label].values)\n",
    "    val_data = lgb.Dataset(train_data.iloc[val_idx][train_features], label=train_data.iloc[val_idx][train_label].values) \n",
    "    \n",
    "    clf = lgb.train(params, trn_data, num_boost_round=2000,valid_sets=[trn_data,val_data], verbose_eval=50, early_stopping_rounds=100, feval=lgb_logloss) \n",
    "    models.append(clf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_res = 0\n",
    "fold = 5\n",
    "for model in models:\n",
    "    pred_res +=model.predict(test_submit[train_features]) * 1.0 / fold "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_submit['prob0'] = 0\n",
    "test_submit['prob1'] = 0\n",
    "test_submit['prob2'] = 0\n",
    "test_submit['prob3'] = 0\n",
    "test_submit['prob4'] = 0\n",
    "test_submit['prob5'] = 0\n",
    "test_submit['prob6'] = 0\n",
    "test_submit['prob7'] = 0\n",
    "\n",
    "test_submit[['prob0','prob1','prob2','prob3','prob4','prob5','prob6','prob7']] = pred_res\n",
    "test_submit[['file_id','prob0','prob1','prob2','prob3','prob4','prob5','prob6','prob7']].to_csv('baseline2.csv',index = None)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:all] *",
   "language": "python",
   "name": "conda-env-all-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "279.273px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
